import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;

import com.google.gdata.client.*;
import com.google.gdata.client.youtube.*;
import com.google.gdata.data.*;
import com.google.gdata.data.youtube.*;
import com.google.gdata.util.*;
import com.hp.hpl.jena.datatypes.RDFDatatype;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.vocabulary.DCTerms;
import com.hp.hpl.jena.vocabulary.DCTypes;
import com.hp.hpl.jena.vocabulary.VCARD;

import java.io.IOException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import com.sun.syndication.feed.synd.SyndEntryImpl;
import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.io.SyndFeedInput;
import com.sun.syndication.io.XmlReader;
public class DBTubeMain
{

	private static DBTubeItem currentDBTubeItem;
	
	private static List<DBTubeItem> DBTubeItemList = new ArrayList<DBTubeItem>(10);
	private static enum Sources {DBPedia,Alchemy,AlchemyCategory };
	
	private static int counter = 0;
	private static int maxAmount = 1000;
	// create an empty Model
	static Model model = ModelFactory.createDefaultModel();
	/**
	 * @param args
	 */
	public static void main(String[] args)
	{

		try
		{
			// Open the file that is the first 
			  // command line parameter
			  FileInputStream fstream = new FileInputStream("../rssFeeds.txt");
			  // Get the object of DataInputStream
			  DataInputStream in = new DataInputStream(fstream);
			  BufferedReader br = new BufferedReader(new InputStreamReader(in));
			  String strLine;
			  //Read File Line By
			  int lineNumber = 1;
			  while ((strLine = br.readLine()) != null && DBTubeItemList.size() < maxAmount)   {
			  // Print the content on the console
				  createDBTubeItems(new URL(strLine));
				  System.out.println("Line "+lineNumber+": "+strLine);
				  lineNumber++;
			  }
			  //Close the input stream
			  in.close();
			  FileWriter fstreamOut = new FileWriter("DBTubeItems3.rdf");
  			BufferedWriter out = new BufferedWriter(fstreamOut);
  			model.write(out);
			out.close();
//			System.out.println("Number of items: "+DBTubeItemList.size());
//			for (DBTubeItem item : DBTubeItemList)
//			{
//				System.out.println(item.toString());
//			}
		}
		catch (Exception e)
		{
			System.out.println(e.getLocalizedMessage());
			e.printStackTrace();
		}
	}

	public static void createDBTubeItems(URL url){
        try {
            SyndFeedInput input = new SyndFeedInput();
            SyndFeed feed = input.build(new XmlReader(url));

	        @SuppressWarnings("unchecked")
			List<SyndEntryImpl> entryList = feed.getEntries();
	        for (SyndEntryImpl syndEntryImpl : entryList)
			{
	        	currentDBTubeItem = new DBTubeItem();
	        	currentDBTubeItem.setTitle(syndEntryImpl.getTitle());
	        	currentDBTubeItem.setContent(syndEntryImpl.getDescription().getValue());
	        	currentDBTubeItem.setNewsSource(new URL(syndEntryImpl.getLink()));
	        	currentDBTubeItem.setAgent(syndEntryImpl.getAuthor());
	        	currentDBTubeItem.setDate(syndEntryImpl.getPublishedDate());
	        	//queryDBPediaSpotlight(currentDBTubeItem.getTitle());
	        	queryAlchemy(currentDBTubeItem.getNewsSource(),"800eefcdbe9d8493647b6f040ee053a971b9ba05");
	        	queryAlchemyCategory(currentDBTubeItem.getNewsSource(),"800eefcdbe9d8493647b6f040ee053a971b9ba05");
	        	String youtubeQuery = currentDBTubeItem.getTitle();
	    		try
	    		{
	    			searchForYoutubeVideo(youtubeQuery);
	    		}
	    		catch (IOException e)
	    		{
	    			// TODO Auto-generated catch block
	    			e.printStackTrace();
	    		}
	    		catch (ServiceException e)
	    		{
	    			// TODO Auto-generated catch block
	    			e.printStackTrace();
	    		}
	    		if(currentDBTubeItem.validate() && !DBTubeItemList.contains(currentDBTubeItem)){
	    			
	    			DBTubeItemList.add(currentDBTubeItem);
	    			Resource dbPediaResource = model.createResource(DBTubeNamespace.Resource);
	    			Resource topicList = model.createResource(DBTubeNamespace.hasTopic);
	    			Resource locationList = model.createResource(DCTerms.Location);
	    			for(DBTubeResource dbTubeResource : currentDBTubeItem.getLocation()){
	    				locationList.addLiteral(DBTubeNamespace.hasName, dbTubeResource.getKeyword());
	    				dbPediaResource.addLiteral(DBTubeNamespace.hasContent, dbTubeResource.getUri());
	    			}
	    			for (DBTubeResource dbTubeResource : currentDBTubeItem.getDBPediaContent()){	
	    				dbPediaResource.addLiteral(DBTubeNamespace.hasContent, dbTubeResource.getUri());
	    				topicList.addLiteral(DBTubeNamespace.hasTopic, dbTubeResource.getKeyword());
	    			}
	    			DateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
	    			model.createResource(DBTubeNamespace.DBTubeItem)
	    			         .addProperty(DBTubeNamespace.hasContent, currentDBTubeItem.getContent())
	    			         .addProperty(DBTubeNamespace.hasTitle, currentDBTubeItem.getTitle())
	    			         .addProperty(DCTerms.creator, model.createResource(DCTerms.Agent)
	    			        		 	.addLiteral(VCARD.FN, currentDBTubeItem.getAgent())
	    			        		 	.addProperty(DBTubeNamespace.hasLocation, locationList))
	    			         .addProperty(DBTubeNamespace.hasRelated, model.createResource(DBTubeNamespace.Video)
	    	                           .addLiteral(DBTubeNamespace.hasContent, currentDBTubeItem.getRelatedVideo())
	    	                           .addLiteral(DBTubeNamespace.isRelated, currentDBTubeItem.getNewsSource())
	    	                 .addProperty(DBTubeNamespace.hasRelated, dbPediaResource )
	    	                 .addLiteral(DCTerms.date, format.format(currentDBTubeItem.getDate()))//Change date type in rdf to http://www.w3.org/2001/XMLSchema#dateTime
	    	                 .addProperty(DBTubeNamespace.hasTopic, topicList)
	    	                 .addLiteral(DCTerms.subject, currentDBTubeItem.getCategory())
	    	                 );
	    			
	    			counter++;
	    			System.out.println("Item #"+counter);
	    			if(counter > maxAmount)
	    				return;
	    		}
	    		
	    		
//	        	System.out.println(syndEntryImpl.getTitle());
//	        	System.out.println(syndEntryImpl.getDescription().getValue());
//	        	System.out.println();
			}
        }
        catch (Exception ex) {
            ex.printStackTrace();
            System.out.println("ERROR: "+ex.getLocalizedMessage());
        }
	}
	
	public static void queryDBPediaSpotlight(String textToQuery){
		try {
		    // Construct data
		    String data = URLEncoder.encode("text", "UTF-8") + "=" + URLEncoder.encode(textToQuery, "UTF-8");
		    data += "&" + URLEncoder.encode("support", "UTF-8") + "=" + URLEncoder.encode("20", "UTF-8");
		    data += "&" + URLEncoder.encode("confidence", "UTF-8") + "=" + URLEncoder.encode("0.1", "UTF-8");
		    //System.out.println(data);
		    // Send data
		    URL url = new URL("http://spotlight.dbpedia.org/rest/annotate");

		   // System.out.println(url);
		    URLConnection conn = url.openConnection();
		    conn.setRequestProperty("Accept", "text/xml");
		    conn.setDoOutput(true);
		    OutputStreamWriter wr = new OutputStreamWriter(conn.getOutputStream());
		    wr.write(data);
		    wr.flush();

		    // Get the response
		    BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
		    String line;
		    FileWriter fstream = new FileWriter("resources.xml");
		    BufferedWriter out = new BufferedWriter(fstream);
		    while ((line = rd.readLine()) != null) {
		       // System.out.println(line);
		        out.write(line);
		    }
		    
		    out.close();
		    wr.close();
		    rd.close();
		    
		    parseXmlFile("resources.xml",Sources.DBPedia);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	private static void queryAlchemyCategory(URL url, String apiKey){
		try
		{
			URL queryURL = new URL("http://access.alchemyapi.com/calls/url/URLGetCategory?apikey="+apiKey+"&url="+url.toExternalForm());
			URLConnection conn = queryURL.openConnection();
		    conn.setRequestProperty("Accept", "text/xml");
		    conn.setDoOutput(true);

		    // Get the response
		    BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
		    String line;
		    FileWriter fstream = new FileWriter("AlchemyCategories.xml");
		    BufferedWriter out = new BufferedWriter(fstream);
		    while ((line = rd.readLine()) != null) {
		       // System.out.println(line);
		        out.write(line);
		    }
		    
		    out.close();
		   // wr.close();
		    rd.close();
		    parseXmlFile("AlchemyCategories.xml",Sources.AlchemyCategory);
		}
		catch (MalformedURLException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	public static void queryAlchemy(URL url, String apiKey){
		try
		{
			URL queryURL = new URL("http://access.alchemyapi.com/calls/url/URLGetRankedNamedEntities?apikey="+apiKey+"&url="+url.toExternalForm());
			URLConnection conn = queryURL.openConnection();
		    conn.setRequestProperty("Accept", "text/xml");
		    conn.setDoOutput(true);

		    // Get the response
		    BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
		    String line;
		    FileWriter fstream = new FileWriter("AlchemyResources.xml");
		    BufferedWriter out = new BufferedWriter(fstream);
		    while ((line = rd.readLine()) != null) {
		       // System.out.println(line);
		        out.write(line);
		    }
		    
		    out.close();
		   // wr.close();
		    rd.close();
		    parseXmlFile("AlchemyResources.xml",Sources.Alchemy);
		}
		catch (MalformedURLException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	public static void searchForYoutubeVideo(String stringQuery) throws IOException, ServiceException{
		YouTubeService service = new YouTubeService("AI39si47dQnKnYwVFXCV4NQ_1vo3fW5sfIrqkOdKtL8aiNz4oiyJeT0Pt4VXf1zKpO-vohv9NLFuYntPKq525N_Bj-h8e7o0tw");
		YouTubeQuery query = new YouTubeQuery(new URL("http://gdata.youtube.com/feeds/api/videos"));
		// order results by the number of views (most viewed first)
		query.setLanguageRestrict("en");
		Query.CategoryFilter filter = new Query.CategoryFilter();
		filter.addCategory(new Category(YouTubeNamespace.CATEGORY_SCHEME, "News"));
		query.addCategoryFilter(filter);

		query.setFullTextQuery(stringQuery);
		query.setSafeSearch(YouTubeQuery.SafeSearch.MODERATE);
		query.setOrderBy(YouTubeQuery.OrderBy.RELEVANCE);

		VideoFeed videoFeed = service.query(query, VideoFeed.class);
		List<VideoEntry> videoEntries = videoFeed.getEntries();
		if(videoEntries != null && videoEntries.size() > 0){
			 //System.out.println("Title: " +videoEntries.get(0).getTitle().getPlainText());
			 currentDBTubeItem.setRelatedVideo(new URL(videoEntries.get(0).getHtmlLink().getHref()));
		}
	}
	
	private static void parseXmlFile(String location, Sources source){
		//get the factory
		DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
		try {

			//Using factory get an instance of document builder
			DocumentBuilder db = dbf.newDocumentBuilder();

			//parse using builder to get DOM representation of the XML file
			Document dom = db.parse(location);
			switch(source){
			case DBPedia:
				parseDocument(dom);
				break;
			case Alchemy:
				parseAlchemyDocument(dom);
				break;
			case AlchemyCategory:
				parseAclhemyCategory(dom);
				break;
			}
		}catch(ParserConfigurationException pce) {
			pce.printStackTrace();
		}catch(SAXException se) {
			se.printStackTrace();
		}catch(IOException ioe) {
			ioe.printStackTrace();
		}
	}
	private static void parseAclhemyCategory(Document dom){
		Element docEle = dom.getDocumentElement();

		//get a nodelist of elements
		currentDBTubeItem.setCategory(getTextValue(docEle, "category"));
	}
	private static void parseAlchemyDocument(Document dom){
		//get the root element
		Element docEle = dom.getDocumentElement();

		//get a nodelist of elements
		NodeList nl = docEle.getElementsByTagName("entity");
		if(nl != null && nl.getLength() > 0) {
			for(int i = 0 ; i < nl.getLength();i++) {
				//get the employee element
				Element el = (Element)nl.item(i);

				//get the Resource object
				try
				{
					getAlchemyEntry(el);
				}
				catch (MalformedURLException e)
				{
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
		}
	}
	
	private static void parseDocument(Document dom){
		//get the root element
		Element docEle = dom.getDocumentElement();

		//get a nodelist of elements
		NodeList nl = docEle.getElementsByTagName("Resource");
		if(nl != null && nl.getLength() > 0) {
			for(int i = 0 ; i < nl.getLength();i++) {
				//get the employee element
				Element el = (Element)nl.item(i);

				//get the Resource object
				try
				{
					currentDBTubeItem.addDBPediaContent(getResource(el));
				}
				catch (MalformedURLException e)
				{
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
		}
	}
	
	/**
	 * I take a xml element and the tag name, look for the tag and get
	 * the text content
	 * i.e for <employee><name>John</name></employee> xml snippet if
	 * the Element points to employee node and tagName is 'name' I will return John
	 */
	private static String getTextValue(Element ele, String tagName) {
		String textVal = null;
		NodeList nl = ele.getElementsByTagName(tagName);
		if(nl != null && nl.getLength() > 0) {
			Element el = (Element)nl.item(0);
			textVal = el.getFirstChild().getNodeValue();
		}

		return textVal;
	}
	
	private static void getAlchemyEntry(Element empEl) throws MalformedURLException {
		ArrayList<String> locations = new ArrayList<String>(Arrays.asList("Country","City","Continent","Region","StateOrCounty"));
		String type = getTextValue(empEl,"type");
		if(locations.contains(type)){
			NodeList nl = empEl.getElementsByTagName("disambiguated");
			if(nl != null && nl.getLength() > 0){
				for(int i = 0; i < nl.getLength(); i++)
				{
					Element el = (Element)nl.item(i);
					String name = getTextValue(empEl,"name");
					String dbpedia = getTextValue(empEl,"dbpedia");
					currentDBTubeItem.addLocation(new DBTubeResource(name, new URL(dbpedia)));
				}
			}
		}
		else {
			NodeList nl = empEl.getElementsByTagName("disambiguated");
			if(nl != null && nl.getLength() > 0){
				for(int i = 0; i < nl.getLength(); i++)
				{
					Element el = (Element)nl.item(i);
					String name = getTextValue(empEl,"name");
					String dbpedia = getTextValue(empEl,"dbpedia");
					currentDBTubeItem.addDBPediaContent(new DBTubeResource(name, new URL(dbpedia)));
				}
			}
		}
	}
	
	private static DBTubeResource getResource(Element empEl) throws MalformedURLException {

		URL uri = new URL(empEl.getAttribute("URI"));
		String keyWord = empEl.getAttribute("surfaceForm");
		DBTubeResource resource = new DBTubeResource(keyWord, uri);
		//System.out.println(resource);
		return resource;
	}
}
